Stable Diffusion

Note

Install ekorpkit package first.

Set logging level to Warning, if you don’t want to see verbose logging.

If you run this notebook in Colab, set Hardware accelerator to GPU.

!pip install ekorpkit[art] exit()

Prepare Environment

%config InlineBackend.figure_format='retina'
%load_ext autotime

from ekorpkit import eKonf

eKonf.setLogger("INFO")
eKonf.set_cuda(device=0)
print("version:", eKonf.__version__)

is_colab = eKonf.is_colab()
print("is colab?", is_colab)
if is_colab:
    eKonf.mount_google_drive()
project_dir = eKonf.set_workspace(workspace="/content/drive/MyDrive/workspace/", project="ekorpkit-book")
print("project_dir:", project_dir)

INFO:ekorpkit.base:Setting cuda device to ['A100-SXM4-40GB (id:0)']
INFO:ekorpkit.utils.notebook:Google Colab not detected.
INFO:ekorpkit.base:Setting EKORPKIT_WORKSPACE_ROOT to /content/drive/MyDrive/workspace/
INFO:ekorpkit.base:Setting EKORPKIT_PROJECT to ekorpkit-book
INFO:ekorpkit.base:Loaded .env from /workspace/projects/ekorpkit-book/config/.env

version: 0.1.40.post0.dev5
is colab? False
project_dir: /content/drive/MyDrive/workspace/projects/ekorpkit-book
time: 2.24 s (started: 2022-11-04 04:15:08 +00:00)

Create a stable diffusion instance

from ekorpkit.models.art.stable import StableDiffusion

# cfg = eKonf.compose("model/stable_diffusion")
# sd = eKonf.instantiate(cfg)
sd = StableDiffusion()

INFO:absl:Unable to initialize backend 'tpu_driver': NOT_FOUND: Unable to find driver in registry given worker: 
INFO:absl:Unable to initialize backend 'rocm': NOT_FOUND: Could not find registered platform with name: "rocm". Available platform names are: CUDA Host Interpreter
INFO:absl:Unable to initialize backend 'tpu': module 'jaxlib.xla_extension' has no attribute 'get_tpu_client'
INFO:ekorpkit.batch:Batch name: stable-batch, Batch num: 0
INFO:ekorpkit.batch:Batch name: stable-batch, Batch num: 0
INFO:ekorpkit.models.art.base:> downloading models...
INFO:ekorpkit.models.art.base:> loading modules...
INFO:ekorpkit.models.art.base:No modules to load
INFO:ekorpkit.models.art.base:> loading models...
INFO:ekorpkit.base:Loaded .env from /workspace/projects/ekorpkit-book/config/.env

INFO:ekorpkit.base:Loaded .env from /workspace/projects/ekorpkit-book/config/.env

time: 56.2 s (started: 2022-11-04 04:15:10 +00:00)

Generate images

text_prompts = "a living tiger inside a gorgeous cave in a futuristic Jeju island, relaxing night, watercolour on the ceiling, glowing on glass, light art, dark, dark, gorgeous, dark, bright in an abandoned village in a futuristic alien nightscape"
batch_name = "cave-jeju"

time: 576 µs (started: 2022-11-04 04:16:06 +00:00)

eKonf.setLogger("INFO")
sd.verbose = True

text_prompts = "examining documents and detailed white background, elegant detailed digital painting, concept art, matte, sharp focus, 4k ultra hd, illustration, magical, unreal engine, octane render, cinematic lighting, wide angle establishing shot, extremely high detail"
batch_name = "lecture"

results = sd.imagine(
    text_prompts, 
    batch_name=batch_name, 
    num_samples=4, 
    guidance_scale=9,
    width=800,
    height=400,
    num_inference_steps=150,
)

INFO:ekorpkit.batch:Using existing path: /content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen
INFO:ekorpkit.batch:Batch name: lecture, Batch num: 1
INFO:ekorpkit.batch:Merging config with args: {}
INFO:ekorpkit.batch:Setting seed to 4074043219
INFO:ekorpkit.models.art.base:Prompt: examining documents and detailed white background, elegant detailed digital painting, concept art, matte, sharp focus, 4k ultra hd, illustration, magical, unreal engine, octane render, cinematic lighting, wide angle establishing shot, extremely high detail

time: 1min 4s (started: 2022-11-04 04:16:06 +00:00)

sd.config.path

{'name': 'disco-imagen', 'root': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen'), 'batch_name': 'demo', 'library_dir': '${mkdir:${join_path:${.root},libs}}', 'cache_dir': '${mkdir:${join_path:${.root},cache}}', 'model_dir': '${mkdir:${join_path:${.root},models}}', 'output_dir': '${mkdir:${join_path:${.root},outputs/stable-diffusion}}', 'batch_dir': '${mkdir:${.output_dir}/${.batch_name}}', 'batch_pair_dir': '${mkdir:${.output_dir}/batch_pairs}', 'tmp_dir': '${mkdir:${join_path:${.root},tmp}}', 'verbose': '${oc.select:..verbose,false}', 'batch': {'name': '${..batch_name}', 'base_dir': '${..batch_dir}'}}

time: 4.44 ms (started: 2022-11-04 04:19:42 +00:00)

Inpaint images

base_url = "https://github.com/entelecheia/ekorpkit-book/raw/main/assets/figs"
init_image_path = f"{base_url}/chu-horse.png"
mask_image_path = f"{base_url}/chu-horse-masking-rough.png"
mask_image_exact_path = f"{base_url}/chu-horse-masking-exact.png"

init_image = eKonf.load_image(init_image_path)
mask_image = eKonf.load_image(mask_image_path)
mask_image_exact = eKonf.load_image(mask_image_exact_path)

sd.compare_images([mask_image, init_image, mask_image_exact], resize_ratio=0.5)

time: 2.53 s (started: 2022-11-03 10:57:07 +00:00)

prompt = "a golden statue of an eagle with clouds, colorful painting by Tanya Hern, mural on the roof, digital art, artwork, beautiful, colorful, visual art on landscape, surrealism, watercolor, vivid by stunning"

results = sd.imagine(
    prompt, 
    mode="inpaint",
    init_image=init_image_path,
    mask_image=mask_image_path,
    num_samples=4,
    num_images_per_prompt=2, 
    inpaint_strength=10, 
    num_inference_steps=100,
    return_including_init_image=False
)

INFO:ekorpkit.batch:Using existing path: /content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen
INFO:ekorpkit.batch:Batch name: cave-jeju, Batch num: 1
INFO:ekorpkit.batch:Merging config with args: {}
INFO:ekorpkit.batch:Setting seed to 2776875379
INFO:ekorpkit.models.art.base:Prompt: a golden statue of an eagle with clouds, colorful painting by Tanya Hern, mural on the roof, digital art, artwork, beautiful, colorful, visual art on landscape, surrealism, watercolor, vivid by stunning

INFO:ekorpkit.batch:Saving config to /content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/stable-diffusion/stable-batch/config/cave-jeju(1)_config.yaml

time: 54 s (started: 2022-11-03 10:57:10 +00:00)

Stitch images

results = sd.imagine(
    text_prompts, 
    mode="stitch",
    batch_name=batch_name, 
    num_samples=6, 
    num_images_per_prompt=2,
    guidance_scale=7,
    inpaint_strength=10, 
)

INFO:ekorpkit.utils.notebook:shell type: ZMQInteractiveShell

INFO:ekorpkit.models.art.stable: >> elapsed time to imagine: 0:02:49.117101
INFO:ekorpkit.batch:Saving config to /content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/stable-diffusion/stable-batch/config/cave-jeju(2)_config.yaml

time: 2min 49s (started: 2022-11-03 10:58:04 +00:00)

To view the panorama, upload the image to renderstuff.

Show Configuration

sd.show_config(batch_name=batch_name, batch_num=0)

INFO:ekorpkit.batch:Using existing path: /content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen
INFO:ekorpkit.batch:Batch name: cave-jeju, Batch num: 0
INFO:ekorpkit.batch:Loading config from /content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/stable-diffusion/stable-batch/config/cave-jeju(0)_config.yaml
INFO:ekorpkit.batch:Merging config with the loaded config
INFO:ekorpkit.batch:Merging config with args: {}
INFO:ekorpkit.batch:Setting seed to 1894816122

{'_target_': 'ekorpkit.models.art.stable.StableDiffusion',
 'autoload': True,
 'batch': {'batch_name': 'cave-jeju',
           'batch_num': 0,
           'resume_run': False,
           'run_to_resume': 'latest',
           'seed': 1894816122,
           'set_seed': 'random_seed'},
 'collage': {'bg_color': 'black', 'cols': -1, 'max_images': 6, 'padding': 5},
 'device': 'cuda:7',
 'hf_user_access_token': None,
 'imagine': {'display_collage': True,
             'display_image': True,
             'guidance_scale': 7,
             'height': 512,
             'increase_seed_by': 1,
             'init_image': None,
             'inpaint_strength': 7.5,
             'make_collage': True,
             'mask_image': None,
             'mode': 'generate',
             'num_images_per_prompt': 1,
             'num_inference_steps': 100,
             'num_samples': 2,
             'save_collage': True,
             'save_image': True,
             'save_image_config': True,
             'seed': None,
             'text_prompts': 'a living tiger inside a gorgeous cave in a '
                             'futuristic Jeju island, relaxing night, '
                             'watercolour on the ceiling, glowing on glass, '
                             'light art, dark, dark, gorgeous, dark, bright in '
                             'an abandoned village in a futuristic alien '
                             'nightscape',
             'width': 768},
 'model': {'generate': {'device': 'cuda:6',
                        'pretrained_model_name_or_path': 'CompVis/stable-diffusion-v1-4',
                        'revision': 'fp16',
                        'use_auth_token': True},
           'inpaint': {'device': 'cuda:5',
                       'pretrained_model_name_or_path': 'runwayml/stable-diffusion-inpainting',
                       'revision': 'fp16',
                       'use_auth_token': True}},
 'module': {'library_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/libs'),
            'modules': None},
 'name': 'stable-diffusion',
 'path': {'batch': {'base_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/stable-diffusion/stable-batch'),
                    'name': 'stable-batch'},
          'batch_configs_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/stable-diffusion/batch_configs'),
          'batch_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/stable-diffusion/stable-batch'),
          'batch_name': 'stable-batch',
          'cache_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/cache'),
          'library_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/libs'),
          'model_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/models'),
          'name': 'disco-imagen',
          'output_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/stable-diffusion'),
          'root': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen'),
          'tmp_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/tmp'),
          'verbose': False},
 'verbose': False}
time: 44.2 ms (started: 2022-11-03 11:00:53 +00:00)

References

• Stable Diffusion

• Diffusers